{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hello\n"
     ]
    }
   ],
   "source": [
    "print('hello')\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 12, 2048])\n",
      "cpu\n",
      "torch.bfloat16\n"
     ]
    }
   ],
   "source": [
    "a = torch.randn(1, 12,2048)\n",
    "print(a.shape)\n",
    "print(a.device)\n",
    "a = a.to(torch.bfloat16)\n",
    "print(a.dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from transformers import AutoTokenizer, AutoModelForCausalLM\n",
    "from gxl_ai_utils.utils import utils_file\n",
    "import os\n",
    "model_path = \"/home/work_nfs9/sywang/ckpt/Qwen2.5-3B-Instruct\"\n",
    "model = AutoModelForCausalLM.from_pretrained(model_path, trust_remote_code=True,torch_dtype=torch.bfloat16,)\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True,)\n",
    "device = torch.device(\"cuda:7\")\n",
    "model.to(device)\n",
    "print(model)\n",
    "\n",
    "def chat(input_q_text):\n",
    "    prompt = input_q_text\n",
    "    messages = [\n",
    "        {\"role\": \"system\", \"content\": \"You are Qwen, created by Alibaba Cloud. You are a helpful assistant.\"},\n",
    "        {\"role\": \"user\", \"content\": prompt}\n",
    "    ]\n",
    "    text = tokenizer.apply_chat_template(\n",
    "        messages,\n",
    "        tokenize=False,\n",
    "        add_generation_prompt=True\n",
    "    )\n",
    "    # print(f'text: {text}')\n",
    "    print(f'text repr: {repr(text)}')\n",
    "    model_inputs = tokenizer([text], return_tensors=\"pt\").to(device)\n",
    "    print(f'model_inputs: {model_inputs.input_ids}')\n",
    "\n",
    "    generated_ids = model.generate(\n",
    "        model_inputs.input_ids,\n",
    "        max_new_tokens=512\n",
    "    )\n",
    "    generated_ids = [\n",
    "        output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)\n",
    "    ]\n",
    "    print(f'generated_ids: {generated_ids}')\n",
    "\n",
    "    response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]\n",
    "    return response\n",
    "\n",
    "# 获取词表大小\n",
    "vocab_size = model.lm_head.weight.shape[1]\n",
    "print(f\"词表大小: {vocab_size}\")\n",
    "vocab_size = model.lm_head.weight.shape[0]\n",
    "print(f\"词表大小: {vocab_size}\")\n",
    "\n",
    "# 获取EOS的ID\n",
    "eos_token_id = tokenizer.eos_token_id\n",
    "print(f\"EOS的ID: {eos_token_id}\")\n",
    "\n",
    "text = \"<|endoftext|>\"\n",
    "id = tokenizer([text], return_tensors=\"pt\").input_ids.to(device)\n",
    "print(f\"text: {text},id: {id}\")\n",
    "\n",
    "id = 151643\n",
    "text = tokenizer.decode(id)\n",
    "print(f\"id: {id},text: {text}\")\n",
    "id = 151645\n",
    "text = tokenizer.decode(id)\n",
    "print(f\"id: {id},text: {text}\")\n",
    "\n",
    "input_q_text = \"中国和日本的关系是什么？\"\n",
    "response = chat(input_q_text)\n",
    "print(\"Assistant:\", response)\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "k2_gxl",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
